Fix escaped quotes and newlines for Android strings (#3650)

Author: eemeli

pontoon/base/migrations/0077_fix_android_newlines.py

@@ -0,0 +1,35 @@
+from re import compile
+
+from django.db import migrations
+
+
+def fix_android_newlines(apps, schema_editor):
+    esc_nl = compile(r"(?<!\\)\\n\s*")
+
+    Entity = apps.get_model("base", "Entity")
+    android_entities = Entity.objects.filter(
+        resource__format="xml", string__contains="\\n"
+    )
+    for ent in android_entities:
+        ent.string = esc_nl.sub(r"\\n\n", ent.string)
+    Entity.objects.bulk_update(android_entities, ["string"])
+
+    Translation = apps.get_model("base", "Translation")
+    android_translations = Translation.objects.filter(
+        entity__resource__format="xml", string__contains="\\n"
+    )
+    for trans in android_translations:
+        trans.string = esc_nl.sub(r"\\n\n", trans.string)
+    Translation.objects.bulk_update(android_translations, ["string"], batch_size=2000)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("base", "0076_remove_pontoon_intro_project"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            code=fix_android_newlines, reverse_code=migrations.RunPython.noop
+        ),
+    ]

pontoon/sync/core/translations_to_repo.py

@@ -358,7 +358,8 @@ def set_translations(
         ]
 
 
-android_quotes = compile(r"(?<!\\)(['\"])")
+android_nl = compile(r"\s*\n\s*")
+android_esc = compile(r"(?<!\\)\\([nt])\s*")
 webext_placeholder = compile(r"\$([a-zA-Z0-9_@]+)\$|(\$[1-9])|\$(\$+)")
 
 
@@ -395,7 +396,11 @@ def set_translation(
     for tx in translations:
         if tx.entity.key == key:
             if res.format == Format.android:
-                entry.value = android_quotes.sub(r"\\\1", tx.string)
+                # Literal newlines \n and tabs \t are included in the string
+                entry.value = android_esc.sub(
+                    lambda m: "\n" if m[1] == "n" else "\t",
+                    android_nl.sub(" ", tx.string),
+                )
             elif (
                 res.format == Format.webext
                 and isinstance(entry.value, PatternMessage)

pontoon/sync/formats/xml.py

@@ -28,7 +28,8 @@ def parse(res: Resource[Message]):
 
 
 esc_u = compile(r"(?<!\\)\\u[0-9]{4}")
-esc_char = compile(r"(?<!\\)\\(.)")
+esc_char = compile(r"(?<!\\)\\([^nt])")
+esc_nl = compile(r"(?<!\\)\\n\s*")
 ws_around_outer_tag = compile(r"^\s+(?=<)|(?<=>)\s+$")
 ws_before_block = compile(r"\s+(?=<(br|label|li|p|/?ul)\b)")
 ws_after_block = compile(r"((?<=<br>)|(?<=<br/>)|(?<=</ul>)|(?<=\\n))\s+")
@@ -40,6 +41,7 @@ def as_translation(order: int, entry: Entry[Message]):
     string = unescape(string)
     string = esc_u.sub(lambda m: chr(int(m[1])), string)
     string = esc_char.sub(r"\1", string)
+    string = esc_nl.sub(r"\\n\n", string)
     string = ws_around_outer_tag.sub("", string)
     string = ws_before_block.sub("\n", string)
     string = ws_after_block.sub("\n", string)

pontoon/sync/tests/test_e2e.py

@@ -10,7 +10,12 @@
 
 from django.conf import settings
 
-from pontoon.base.models import ChangedEntityLocale, TranslatedResource, Translation
+from pontoon.base.models import (
+    ChangedEntityLocale,
+    Entity,
+    TranslatedResource,
+    Translation,
+)
 from pontoon.base.tests import (
     EntityFactory,
     LocaleFactory,
@@ -231,6 +236,78 @@ def test_translation_before_source():
             assert file.read() == "a0 = New translation 0\n"
 
 
+@pytest.mark.django_db
+def test_android():
+    mock_vcs = MockVersionControl(changes=None)
+    with (
+        TemporaryDirectory() as root,
+        patch("pontoon.sync.core.checkout.get_repo", return_value=mock_vcs),
+        patch("pontoon.sync.core.translations_to_repo.get_repo", return_value=mock_vcs),
+    ):
+        # Database setup
+        settings.MEDIA_ROOT = root
+        locale = LocaleFactory.create(code="de-Test", name="Test German")
+        repo_src = RepositoryFactory(
+            url="http://example.com/src-repo", source_repo=True
+        )
+        repo_tgt = RepositoryFactory(url="http://example.com/tgt-repo")
+        project = ProjectFactory.create(
+            name="test-android",
+            locales=[locale],
+            repositories=[repo_src, repo_tgt],
+        )
+        res = ResourceFactory.create(project=project, path="strings.xml", format="xml")
+
+        entity = EntityFactory.create(resource=res, key="quotes", string="Prev quotes")
+        TranslationFactory.create(
+            entity=entity,
+            locale=locale,
+            string="'Hello' \"translation\"",
+            active=True,
+            approved=True,
+        )
+
+        entity = EntityFactory.create(
+            resource=res, key="newline", string="Prev newline"
+        )
+        TranslationFactory.create(
+            entity=entity,
+            locale=locale,
+            string="translated \n escaped \\n newlines",
+            active=True,
+            approved=True,
+        )
+
+        # Filesystem setup
+        src_root = repo_src.checkout_path
+        src_xml = """<?xml version="1.0" ?>
+            <resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+                <string name="quotes">\\'Hello\\' \\"source\\"</string>
+                <string name="newline">literal \n escaped \\n newlines</string>
+            </resources>"""
+        makedirs(src_root)
+        build_file_tree(src_root, {"en-US": {"strings.xml": src_xml}})
+
+        tgt_root = repo_tgt.checkout_path
+        makedirs(tgt_root)
+        build_file_tree(tgt_root, {"de-Test": {}})
+
+        # Test
+        sync_project_task(project.pk)
+        assert set(ent.string for ent in Entity.objects.filter(resource=res)) == {
+            "'Hello' \"source\"",
+            "literal escaped \\n\nnewlines",
+        }
+        with open(join(repo_tgt.checkout_path, "de-Test", "strings.xml")) as file:
+            assert file.read() == dedent("""\
+                <?xml version="1.0" encoding="utf-8"?>
+                <resources>
+                  <string name="quotes">\\'Hello\\' \\"translation\\"</string>
+                  <string name="newline">translated escaped \\nnewlines</string>
+                </resources>
+                """)
+
+
 @pytest.mark.django_db
 def test_fuzzy():
     with TemporaryDirectory() as root: